package com.dtruth.dataflow.bolt;

import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.commons.lang3.exception.ExceptionUtils;
import org.slf4j.Logger;

import backtype.storm.task.TopologyContext;
import backtype.storm.topology.BasicOutputCollector;
import backtype.storm.topology.OutputFieldsDeclarer;
import backtype.storm.topology.base.BaseBasicBolt;
import backtype.storm.tuple.Fields;
import backtype.storm.tuple.Tuple;
import backtype.storm.tuple.Values;

import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.dtruth.dataflow.util.HtmlRegexpUtil;
import com.dtruth.dataflow.util.MyLoggerFactory;

public class BodyReformatBolt extends BaseBasicBolt {

	private static Logger log = MyLoggerFactory
			.getLogger(BodyReformatBolt.class);

	private static final long serialVersionUID = 1L;

	@Override
	public void prepare(Map stormConf, TopologyContext context) {
	}

	@Override
	public void execute(Tuple input, BasicOutputCollector collector) {
		@SuppressWarnings("unchecked")
		Map<String, Object> map = (Map<String, Object>) input.getValue(0);
		String body = process(map);
		if (body != null && body.length() > 150) {
			map.put("body", body);
			log.debug("emit:" + map.get("url"));
			collector.emit(new Values(map));
		} else {
			log.warn("body is too short after reformatting:" + map.get("url"));
		}

	}

	public String process(Map<String, Object> map) {
		String body = null;
		try {
			body = removeAddImg(map);
			if (map.get("domain") != null)
				body = specialHandle(body, map.get("domain").toString());
			body = reformat(body);
		} catch (Exception e) {
			log.error(ExceptionUtils.getStackTrace(e) + ":" + map.get("url"));
		}
		return body;
	}

	public String removeAddImg(Map<String, Object> map) {
		Object obj = map.get("body");
		String body = obj != null ? obj.toString() : "";
		Set<String> imgAddrs = new HashSet<String>();
		body = HtmlRegexpUtil.removeImgWithLink(body, imgAddrs);
		obj = map.get("thumbnail");
		if (obj != null) {
			JSONArray jarry = JSONObject.parseArray(obj.toString());
			for (String imgAddr : imgAddrs) {
				jarry.remove(imgAddr);
			}
			map.put("thumbnail", jarry);
		}
		return body;
	}

	private String specialHandle(String body, String domain) {
		if (domain.indexOf("fang.com") > -1) {
			String begin = "<!--wirelessHiddenContentBegin-->";
			String end = "<!--wirelessHiddenContentEnd-->";
			int b = body.indexOf(begin);
			int e = body.indexOf(end);
			if (b > -1 && e > -1) {
				String subStr = body.substring(b, e + end.length());
				return body.replaceAll(subStr, "");
			}
		}
		if (domain.indexOf("21cbh.com") > -1) {
			String regxp = "(</*a[^>]*?>返回21世纪网首页&gt;&gt;</a>)";
			Pattern pattern = Pattern.compile(regxp, Pattern.CASE_INSENSITIVE);
			Matcher matcher = pattern.matcher(body);
			while (matcher.find()) {
				String fStr = matcher.group();
				body = body.replace(fStr, "");
			}
			return body;
		}
		if (domain.indexOf("cnbm.net.cn") > -1) {
			String regxp = "(<div[^>]*?>((?!</div>).)+</div>)";
			Pattern pattern = Pattern.compile(regxp, Pattern.CASE_INSENSITIVE);
			Matcher matcher = pattern.matcher(body);
			while (matcher.find()) {
				String fStr = matcher.group();
				if (fStr.indexOf("class=\"mouse_rela_course\"") > -1)
					body = body.replace(fStr, "");
			}
			return body;
		}
		if (domain.indexOf("huanqiu.com") > -1) {
			String regxp = "<!-- 翻页按钮 begin -->.*<!-- 翻页按钮 end -->";
			Pattern pattern = Pattern.compile(regxp, Pattern.CASE_INSENSITIVE);
			Matcher matcher = pattern.matcher(body);
			while (matcher.find()) {
				String fStr = matcher.group();
				body = body.replace(fStr, "");
			}
			return body;
		}
		if (domain.indexOf("cebnet.com.cn") > -1) {
			body = HtmlRegexpUtil.removeDisplayNone(body, "div");
		}
		return body;
	}

	public String reformat(String htmlStr) {
		htmlStr = htmlStr.replaceAll("<em>浏览原图</em>", "");
		htmlStr = HtmlRegexpUtil.removeComment(htmlStr);
		htmlStr = HtmlRegexpUtil.removeScript(htmlStr);
		htmlStr = HtmlRegexpUtil.removeStyle(htmlStr);
		htmlStr = HtmlRegexpUtil.removeEmoji(htmlStr);
		Set<String> keepTags = new HashSet<String>();
		keepTags.add("p");
		keepTags.add("img");
		keepTags.add("br");
		keepTags.add("div");
		keepTags.add("ol");
		keepTags.add("ul");
		keepTags.add("li");
		keepTags.add("ul");
		keepTags.add("table");
		keepTags.add("tr");
		keepTags.add("td");
		keepTags.add("strong");
		htmlStr = HtmlRegexpUtil.removePartTags(htmlStr, keepTags);
		htmlStr = HtmlRegexpUtil.removeSpace(htmlStr);
		Set<String> keepTagAttr = new HashSet<String>();
		keepTagAttr.add("img");
		keepTags.removeAll(keepTagAttr);
		for (String keepTag : keepTags) {
			htmlStr = HtmlRegexpUtil.removeTagAllAttrib(htmlStr, keepTag);
		}
		htmlStr = htmlStr.replaceAll("<div\\s*>[\r|\n|\t]*?</div>", "");
		htmlStr = htmlStr.replaceAll("<div\\s*>[^\u4e00-\u9fa5]*?</div>", "");
		htmlStr = htmlStr.replaceAll("<p\\s*>[\r|\n|\t]*?</p>", "");
		htmlStr = htmlStr.replaceAll("<p\\s*><br\\s*>", "<p>");
		htmlStr = htmlStr.replaceAll(
				"<p[^>]*?>[^\u4e00-\u9fa5|^i]*?</p[^>]*?>", "");
		htmlStr = HtmlRegexpUtil.removeSpace(htmlStr);
		return htmlStr.trim(); // 返回文本字符串
	}

	@Override
	public void declareOutputFields(OutputFieldsDeclarer declarer) {
		declarer.declare(new Fields("news"));
	}
}
